{ "hidden_size": 1024, "intermediate_size": 3072, "num_hidden_layers": 28, "num_attention_heads": 16, "num_key_value_heads": 8, "head_dim": 128, "vocab_size": 151677, "mask_vocab_size": 151677, "mask_token_id": 151669, "pad_token_id": 151643, "max_position_embeddings": 40960, "rope_theta": 1000000.0, "rms_norm_eps": 1e-06, "hidden_act": "silu", "qk_norm": true, "timestep_emb_hidden": 1024, "tie_word_embeddings": false, "mask_ratio_min": 0.0, "mask_ratio_max": 1.0, "model_type": "metadiffusion", "architectures": [ "MetaDiffusion600MForCausalLM" ], "auto_map": { "AutoConfig": "hf_modeling.MetaDiffusion600MConfig", "AutoModelForCausalLM": "hf_modeling.MetaDiffusion600MForCausalLM" }, "eos_token_id": [ 151645 ], "rainbow_token_ids": [ 151670, 151671, 151672, 151673, 151674, 151675, 151676 ], "invalid_utf8_token_ids": [ 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 378, 1277, 1540, 2194, 2775, 3023, 3219, 3402, 3490, 4811, 4973, 5502, 5564, 5691, 5758, 5902, 5953, 6142, 6518, 6564, 6571, 6606, 6684, 7045, 7158, 7311, 7367, 7492, 7536, 7588, 7908, 8032, 8079, 8158, 8508, 8519, 8843, 9260, 9284, 9722, 9973, 10081, 10203, 10417, 10663, 10996, 11125, 11336, 11484, 11534, 11620, 11621, 12274, 12491, 12585, 12619, 12787, 13058, 13092, 13165, 13344, 13465, 13512, 13519, 13531, 13558, 13567, 13846, 13869, 13949, 13982, 14273, 14368, 14467, 14546, 14559, 14675, 14922, 14936, 14942, 14959, 15005, 15127, 15213, 15224, 15362, 15393, 15600, 15675, 15798, 15827, 15896, 15902, 15927, 16160, 16499, 16633, 16747, 16825, 17049, 17156, 17158, 17401, 17429, 17462, 17467, 17641, 17730, 17783, 18059, 18505, 18535, 19088, 19113, 19261, 19457, 19549, 19564, 19741, 19956, 20026, 20047, 20098, 20382, 20741, 20778, 20879, 20885, 20919, 21012, 21135, 21280, 21530, 21562, 21704, 21849, 22009, 22129, 22247, 22597, 22762, 22859, 23093, 23272, 23433, 23444, 23479, 23687, 23781, 24071, 24159, 24345, 24447, 24735, 24749, 24831, 24864, 25145, 25209, 25452, 25703, 25870, 25928, 26188, 26254, 26336, 26454, 26601, 26771, 26927, 27095, 27209, 27214, 27325, 27398, 27517, 27538, 27640, 27757, 27999, 28107, 28145, 28757, 29150, 29220, 29237, 29389, 29675, 29768, 29955, 29975, 30006, 30274, 30395, 30828, 30838, 30916, 31305, 31482, 31501, 32208, 32254, 32343, 32465, 32843, 32926, 33173, 33352, 33543, 33556, 33593, 33739, 33986, 34429, 34577, 34622, 34629, 34759, 34811, 34992, 35017, 35049, 35055, 35146, 35293, 35496, 35635, 36097, 36109, 36677, 36695, 37289, 37440, 37472, 37698, 37913, 38177, 38419, 38522, 38911, 39098, 39175, 39317, 39834, 40419, 40714, 40732, 40747, 41902, 42144, 43410, 43459, 43547, 43559, 43752, 44014, 44044, 44104, 44258, 44401, 44680, 44706, 44726, 44818, 44832, 44851, 44965, 45881, 46282, 46353, 46800, 47378, 47379, 47972, 48312, 48533, 48591, 48749, 48800, 49166, 49173, 49420, 49943, 50111, 51275, 51497, 52133, 52408, 52798, 52887, 53496, 53556, 54128, 54492, 54642, 55890, 56252, 56823, 56842, 57154, 57160, 57332, 57599, 57743, 58098, 58169, 58299, 58557, 58715, 58899, 58994, 60627, 60757, 60864, 62005, 62416, 62544, 63046, 63219, 63761, 64520, 64893, 65185, 65291, 65850, 65877, 66118, 66498, 67895, 68405, 68597, 69382, 69687, 69792, 70179, 70297, 70467, 70731, 71306, 71443, 71481, 71933, 72219, 72496, 72497, 72509, 74165, 74209, 75107, 75142, 75360, 75375, 75598, 75962, 76080, 76986, 77156, 78910, 79142, 79531, 79590, 79888, 80080, 80702, 82501, 82712, 82798, 82912, 83050, 83164, 83191, 83643, 83849, 84200, 84215, 85794, 86643, 87299, 87959, 88541, 89061, 89083, 89179, 89297, 90557, 90639, 90679, 90734, 90839, 90894, 91048, 91771, 91888, 92120, 92173, 94964, 95211, 96938, 97015, 97946, 98642, 98734, 99156, 99157, 99158, 99159, 99160, 99161, 99162, 99163, 99166, 99167, 99168, 99169, 99170, 99171, 99173, 99174, 99175, 99176, 99177, 99179, 99181, 99183, 99184, 99187, 99188, 99189, 99192, 99196, 99197, 99198, 99201, 99202, 99203, 99206, 99207, 99211, 99214, 99215, 99220, 99221, 99224, 99228, 99229, 99230, 99231, 99238, 99239, 99240, 99247, 99248, 99249, 99254, 99255, 99256, 99264, 99265, 99266, 99267, 99268, 99269, 99274, 99275, 99276, 99281, 99282, 99289, 99290, 99291, 99300, 99301, 99302, 99303, 99311, 99323, 99324, 99325, 99341, 99342, 99343, 99344, 99359, 99374, 99380, 99381, 99382, 99401, 99402, 99423, 99439, 99442, 99443, 99444, 99456, 99484, 99485, 99516, 99551, 99552, 99597, 99598, 99643, 99648, 99649, 99714, 99715, 99777, 99785, 99874, 99875, 99977, 99997, 100024, 100120, 100121, 100127, 100128, 100129, 100130, 100337, 100459, 100563, 100620, 100621, 100988, 101018, 101022, 101024, 101025, 101026, 101027, 101028, 101029, 101030, 101031, 101032, 101033, 101759, 101838, 101851, 101860, 101861, 101862, 101863, 101864, 101865, 101866, 101867, 101868, 101869, 101870, 101871, 101872, 101873, 101874, 101875, 101876, 101877, 101878, 101879, 101880, 101881, 103893, 103894, 103895, 103896, 103897, 103898, 103899, 103900, 103901, 103902, 103903, 103904, 103905, 103906, 103907, 103908, 103909, 103910, 103911, 103912, 103913, 103914, 103915, 103916, 103917, 104905, 109992, 119158, 119941, 119964, 120100, 120409, 121404, 121667, 121773, 122154, 122174, 122202, 122219, 122222, 122259, 122267, 122289, 122317, 122375, 122382, 122427, 122448, 122455, 122514, 122568, 122596, 122614, 122618, 122634, 122659, 122725, 122740, 122757, 122802, 122843, 122855, 122866, 122890, 122948, 122950, 122976, 122985, 122999, 123007, 123008, 123016, 123057, 123064, 123067, 123072, 123084, 123091, 123131, 123164, 123178, 123182, 123204, 123206, 123287, 123301, 123305, 123318, 123332, 123378, 123400, 123408, 123420, 123489, 123516, 123527, 123547, 123614, 123658, 123676, 123701, 123714, 123740, 123747, 123760, 123806, 123807, 123808, 123810, 123811, 123812, 123813, 123814, 123815, 123816, 123817, 123819, 123821, 123825, 123827, 123830, 123836, 123837, 123839, 123840, 123841, 123842, 123844, 123846, 123847, 123848, 123851, 123853, 123854, 123859, 123867, 123870, 123871, 123875, 123876, 123882, 123889, 123892, 123893, 123907, 123908, 123911, 123912, 123916, 123917, 123918, 123927, 123928, 123929, 123930, 123931, 123932, 123933, 123934, 123946, 123947, 123950, 123951, 123953, 123954, 123955, 123956, 123957, 123965, 123967, 123968, 123969, 123970, 123986, 123989, 123996, 124000, 124001, 124002, 124003, 124004, 124005, 124018, 124019, 124020, 124021, 124022, 124023, 124024, 124025, 124026, 124049, 124050, 124051, 124052, 124053, 124054, 124063, 124064, 124066, 124067, 124068, 124069, 124070, 124071, 124092, 124096, 124098, 124099, 124100, 124101, 124102, 124118, 124134, 124135, 124136, 124137, 124149, 124154, 124159, 124160, 124161, 124162, 124163, 124164, 124165, 124166, 124167, 124168, 124193, 124194, 124195, 124196, 124197, 124198, 124199, 124200, 124201, 124222, 124235, 124236, 124240, 124241, 124243, 124244, 124245, 124246, 124247, 124248, 124249, 124250, 124251, 124252, 124287, 124296, 124297, 124298, 124299, 124300, 124301, 124302, 124303, 124304, 124305, 124306, 124307, 124308, 124309, 124334, 124349, 124364, 124365, 124366, 124367, 124368, 124369, 124370, 124371, 124372, 124373, 124374, 124433, 124470, 124471, 124472, 124473, 124474, 124475, 124570, 124577, 124578, 124579, 124580, 124581, 124582, 124583, 124584, 124585, 124586, 124587, 124588, 124589, 124590, 124591, 124592, 124593, 124594, 124595, 124596, 124727, 124744, 124745, 124746, 124747, 124748, 124749, 124750, 124751, 124752, 124753, 124754, 124755, 124995, 124996, 124997, 124998, 124999, 125000, 125001, 125002, 125003, 125004, 125413, 125414, 125415, 125416, 125417, 125418, 125419, 125420, 125421, 125422, 125423, 125424, 125425, 125426, 125427, 125428, 125429, 125430, 126159, 126177, 126178, 126179, 126180, 126181, 126182, 126183, 126184, 126185, 126186, 126187, 126188, 126189, 126190, 126191, 126192, 126193, 127964, 128222, 128223, 128224, 128225, 128226, 128227, 128228, 128229, 128230, 128231, 128232, 128233, 128234, 128235, 128236, 128237, 128238, 128239, 128240, 128241, 128242, 145773, 146047, 146632, 148462, 148774, 148827, 148860, 148864, 148880, 149175, 149178, 149287, 149352, 149392, 149394, 149589, 149591, 149593, 149714, 149716, 149718, 149721, 149946, 149961, 149963, 149990, 150114, 150151, 150168, 150195, 150223, 150270, 150479, 150579, 150792, 150794, 150800, 150802, 150807, 150809, 150814, 150817, 150819, 150821, 150824, 150829, 150836, 151233, 151254, 151264, 151266, 151268, 151270, 151272, 151274, 151276, 151278, 151282, 151366, 151560 ] }