{"id":28287,"date":"2023-10-03T12:53:27","date_gmt":"2023-10-03T03:53:27","guid":{"rendered":"http:\/\/nori.company\/?p=28287"},"modified":"2023-10-03T12:53:27","modified_gmt":"2023-10-03T03:53:27","slug":"meta-32k-%ed%86%a0%ed%81%b0%ec%9d%98-%ec%bb%a8%ed%85%8d%ec%8a%a4%ed%8a%b8-%ec%9c%88%eb%8f%84%ec%9a%b0%eb%a5%bc-%ec%a7%80%ec%9b%90%ed%95%98%eb%8a%94-llama-2-long-%ea%b3%b5%ea%b0%9c","status":"publish","type":"post","link":"https:\/\/nori.company\/?p=28287","title":{"rendered":"Meta, 32k \ud1a0\ud070\uc758 \ucee8\ud14d\uc2a4\ud2b8 \uc708\ub3c4\uc6b0\ub97c \uc9c0\uc6d0\ud558\ub294 LLAMA 2 Long \uacf5\uac1c"},"content":{"rendered":"<ul>\n<li>\ud30c\uc6b4\ub370\uc774\uc158 \ubaa8\ub378\uc758 \ud6a8\uacfc\uc801\uc778 \uc7a5\uae30 \ucee8\ud14d\uc2a4\ud2b8 \ud655\uc7a5(Effective Long-Context Scaling of Foundation Models)\uc774\ub77c\ub294 \ub17c\ubb38\uc744 \ud1b5\ud574 LLaMA2 Long \ubaa8\ub378\uc744 \uacf5\uac1c\n<ul>\n<li>\ub17c\ubb38 \ub9c1\ud06c: https:\/\/ift.tt\/ZWq2Ih8<\/li>\n<\/ul>\n<\/li>\n<li>\ucee8\ud14d\uc2a4\ud2b8 \uc708\ub3c4\uc6b0(context window)\uc758 \uae38\uc774\uac00 32K(32,768) \ud1a0\ud070\uae4c\uc9c0 \uc9c0\uc6d0<\/l...\n\n\n\ucd9c\ucc98 : <a href=\"https:\/\/news.hada.io\/topic?id=11169\" target=\"_blank\" rel=\"noopener\">GeekNews &#8211; \uac1c\ubc1c\/\uae30\uc220\/\uc2a4\ud0c0\ud2b8\uc5c5 \ub274\uc2a4 \uc11c\ube44\uc2a4 <\/a><\/li>\n<\/ul>\n","protected":false},"excerpt":{"rendered":"<p>\ud30c\uc6b4\ub370\uc774\uc158 \ubaa8\ub378\uc758 \ud6a8\uacfc\uc801\uc778 \uc7a5\uae30 \ucee8\ud14d\uc2a4\ud2b8 \ud655\uc7a5(Effective Long-Context Scaling of Foundation Models)\uc774\ub77c\ub294 \ub17c\ubb38\uc744 \ud1b5\ud574 LLaMA2 Long \ubaa8\ub378\uc744 \uacf5\uac1c \ub17c\ubb38 \ub9c1\ud06c: https:\/\/ift.tt\/ZWq2Ih8&hellip;<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"jetpack_post_was_ever_published":false,"_jetpack_newsletter_access":"","_jetpack_dont_email_post_to_subs":false,"_jetpack_newsletter_tier_id":0,"_jetpack_memberships_contains_paywalled_content":false,"_jetpack_memberships_contains_paid_content":false,"footnotes":"","jetpack_publicize_message":"","jetpack_publicize_feature_enabled":true,"jetpack_social_post_already_shared":true,"jetpack_social_options":{"image_generator_settings":{"template":"highway","default_image_id":0,"font":"","enabled":false},"version":2}},"categories":[138],"tags":[163],"class_list":["post-28287","post","type-post","status-publish","format-standard","hentry","category-it","tag-geeknews-----"],"jetpack_publicize_connections":[],"jetpack_featured_media_url":"","jetpack_sharing_enabled":true,"jetpack_shortlink":"https:\/\/wp.me\/pa3PO4-7mf","jetpack-related-posts":[{"id":22544,"url":"https:\/\/nori.company\/?p=22544","url_meta":{"origin":28287,"position":0},"title":"StableLM \uacf5\uac1c &#8211; Stability AI\uc758 \uacf5\uac1c \uc5b8\uc5b4 \ubaa8\ub378","author":"\ub178\ub9ac\ucef4\ud37c\ub2c8","date":"2023\ub144 4\uc6d4 20\uc77c","format":false,"excerpt":"3B\/7B \ubaa8\ub378\uc744 \uacf5\uac1c, 15B\/30B\/65B \ubaa8\ub378\ub3c4 \uacf5\uac1c \uc608\uc815\uc774\uace0 175B\uae4c\uc9c0 \uacc4\ud68d\uc911 \ubaa8\ub378\uc740 CC BY-SA-4.0 \ub77c\uc774\uc13c\uc2a4\ub85c \ucd9c\ucc98 \ud45c\uae30\uc2dc \uc0c1\uc5c5\uc801 \uc774\uc6a9 \uac00\ub2a5 \uc624\ud508 \ub370\uc774\ud130\uc14b\uc778 The Pile\uc5d0 \uae30\ubc18\ud588\uc9c0\ub9cc 3\ubc30 \ud06c\uae30\uc778 1.5T \ud1a0\ud070\uc744 \uac00\uc9c0\ub294 \uc0c8\ub85c\uc6b4 \ub370\uc774\ud130\uc14b\uc73c\ub85c \ud6c8\ub828 \ucee8\ud14d\uc2a4\ud2b8 \uae38\uc774\ub294 4096 \ud1a0\ud070 PoC\ub85c... \ucd9c\ucc98 : GeekNews - \uac1c\ubc1c\/\uae30\uc220\/\uc2a4\ud0c0\ud2b8\uc5c5 \ub274\uc2a4 \uc11c\ube44\uc2a4","rel":"","context":"&quot;IT&quot;\uc5d0\uc11c","block_context":{"text":"IT","link":"https:\/\/nori.company\/?cat=138"},"img":{"alt_text":"","src":"","width":0,"height":0},"classes":[]},{"id":25173,"url":"https:\/\/nori.company\/?p=25173","url_meta":{"origin":28287,"position":1},"title":"MosaicML, MPT-7B-8K \ubaa8\ub378 \uacf5\uac1c","author":"\ub178\ub9ac\ucef4\ud37c\ub2c8","date":"2023\ub144 7\uc6d4 21\uc77c","format":false,"excerpt":"8k \ucee8\ud14d\uc2a4\ud2b8 \uae38\uc774\ub97c \uc9c0\uc6d0\ud558\ub294 7B \ud30c\ub77c\ubbf8\ud130 \uc624\ud508\uc18c\uc2a4 LLM MPT-7B\uc5d0 \ucd94\uac00\ub85c 500B \ud1a0\ud070 \ub370\uc774\ud130\ub97c \uc774\uc6a9\ud574 256\uac1c\uc758 NVidia H100 \uc73c\ub85c 3\uc77c\uac04 \ud2b8\ub808\uc774\ub2dd 3\uac1c\uc758 \ubaa8\ub378\uc744 \uacf5\uac1c : MPT-7B-8k, MPT-7B-8k-Instruct, MPT-7B-8k-Chat \uc0c1\uc5c5\uc801 \uc6a9\ub3c4\ub85c \uc0ac\uc6a9 \uac00\ub2a5 ALiBi(Attention with Linear... \ucd9c\ucc98 : GeekNews - \uac1c\ubc1c\/\uae30\uc220\/\uc2a4\ud0c0\ud2b8\uc5c5 \ub274\uc2a4 \uc11c\ube44\uc2a4","rel":"","context":"&quot;IT&quot;\uc5d0\uc11c","block_context":{"text":"IT","link":"https:\/\/nori.company\/?cat=138"},"img":{"alt_text":"","src":"","width":0,"height":0},"classes":[]},{"id":25088,"url":"https:\/\/nori.company\/?p=25088","url_meta":{"origin":28287,"position":2},"title":"Meta, \ucc28\uc138\ub300 \uc5b8\uc5b4\ubaa8\ub378 Llama 2 \uacf5\uac1c","author":"\ub178\ub9ac\ucef4\ud37c\ub2c8","date":"2023\ub144 7\uc6d4 19\uc77c","format":false,"excerpt":"\uc5f0\uad6c \ubc0f \uc0c1\uc5c5\uc6a9\ub3c4 \ubaa8\ub450 \uc0ac\uc6a9\uac00\ub2a5\ud55c \uc624\ud508\uc18c\uc2a4 \ub300\uaddc\ubaa8 \uc5b8\uc5b4\ubaa8\ub378 Llama 1 \ub300\ube44 40% \ub9ce\uc740 2\uc870\uac1c\uc758 \ud1a0\ud070 \ub370\uc774\ud130\ub85c \ud6c8\ub828 2\ubc30\uc758 \ucee8\ud14d\uc2a4\ud2b8 \uae38\uc774: 4096 \uc138\uac00\uc9c0 \ud06c\uae30\uc758 \ubaa8\ub378: 7B, 13B, 70B \ucd94\ub860, \ucf54\ub529, \uc219\ub828\ub3c4, \uc9c0\uc2dd\ud14c\uc2a4\ud2b8\ub4f1 \ub9ce\uc740 \ubca4\uce58\ub9c8\ud06c\uc5d0\uc11c \ub2e4\ub978 \uc624\ud508\uc18c\uc2a4 \uc5b8\uc5b4 \ubaa8\ub378\ubcf4\ub2e4 \ud6cc\ub96d\ud55c ... \ucd9c\ucc98 : GeekNews - \uac1c\ubc1c\/\uae30\uc220\/\uc2a4\ud0c0\ud2b8\uc5c5 \ub274\uc2a4 \uc11c\ube44\uc2a4","rel":"","context":"&quot;IT&quot;\uc5d0\uc11c","block_context":{"text":"IT","link":"https:\/\/nori.company\/?cat=138"},"img":{"alt_text":"","src":"","width":0,"height":0},"classes":[]},{"id":25092,"url":"https:\/\/nori.company\/?p=25092","url_meta":{"origin":28287,"position":3},"title":"Meta, \ucc28\uc138\ub300 \uc5b8\uc5b4\ubaa8\ub378 Llama 2 \uacf5\uac1c","author":"\ub178\ub9ac\ucef4\ud37c\ub2c8","date":"2023\ub144 7\uc6d4 19\uc77c","format":false,"excerpt":"\uc5f0\uad6c \ubc0f \uc0c1\uc5c5\uc6a9\ub3c4 \ubaa8\ub450 \uc0ac\uc6a9\uac00\ub2a5\ud55c \uc624\ud508\uc18c\uc2a4 \ub300\uaddc\ubaa8 \uc5b8\uc5b4\ubaa8\ub378 Llama 1 \ub300\ube44 40% \ub9ce\uc740 2\uc870\uac1c\uc758 \ud1a0\ud070 \ub370\uc774\ud130\ub85c \ud6c8\ub828 2\ubc30\uc758 \ucee8\ud14d\uc2a4\ud2b8 \uae38\uc774: 4096 \uc138\uac00\uc9c0 \ud06c\uae30\uc758 \ubaa8\ub378: 7B, 13B, 70B \ucd94\ub860, \ucf54\ub529, \uc219\ub828\ub3c4, \uc9c0\uc2dd\ud14c\uc2a4\ud2b8\ub4f1 \ub9ce\uc740 \ubca4\uce58\ub9c8\ud06c\uc5d0\uc11c \ub2e4\ub978 \uc624\ud508\uc18c\uc2a4 \uc5b8\uc5b4 \ubaa8\ub378\ubcf4\ub2e4 \ud6cc\ub96d\ud55c ... \ucd9c\ucc98 : GeekNews - \uac1c\ubc1c\/\uae30\uc220\/\uc2a4\ud0c0\ud2b8\uc5c5 \ub274\uc2a4 \uc11c\ube44\uc2a4","rel":"","context":"&quot;IT&quot;\uc5d0\uc11c","block_context":{"text":"IT","link":"https:\/\/nori.company\/?cat=138"},"img":{"alt_text":"","src":"","width":0,"height":0},"classes":[]},{"id":28486,"url":"https:\/\/nori.company\/?p=28486","url_meta":{"origin":28287,"position":4},"title":"[2023\/10\/02 ~ 10\/08] \uc774\ubc88 \uc8fc\uc758 \uc8fc\uc694 ML \ub17c\ubb38 (Top ML Papers of the Week)","author":"\ub178\ub9ac\ucef4\ud37c\ub2c8","date":"2023\ub144 10\uc6d4 9\uc77c","format":false,"excerpt":"\uac1c\uc694 DAIR.AI\uc5d0\uc11c \ub9e4\uc8fc \uacf5\uac1c\ud558\ub294 ML \ub17c\ubb38\ub4e4\uc5d0 \ub300\ud55c \uae00\uc744 \uc790\ub3d9 \ubc88\uc5ed\ud574\ubcf4\uc558\uc2b5\ub2c8\ub2e4. \uc774\ubc88 \uc8fc\uc5d0 \uc8fc\uc5b4\uc9c4 \ub17c\ubb38\ub4e4\uc744 \uc0b4\ud3b4\ubcf8 \uacb0\uacfc, \uc7a5\uae30\uac04 \ucee8\ud14d\uc2a4\ud2b8(Long Context)\ub97c \ub2e4\ub8e8\ub294 \uc5b8\uc5b4 \ubaa8\ub378(Language Models, LLM)\uc5d0 \ub300\ud55c \uc5f0\uad6c\ub4e4\uc774 \ub9ce\uc558\uc2b5\ub2c8\ub2e4. \ud2b9\ud788 'LLMs Represent Space and Time', 'Retr \ucd9c\ucc98 : GeekNews - \uac1c\ubc1c\/\uae30\uc220\/\uc2a4\ud0c0\ud2b8\uc5c5 \ub274\uc2a4 \uc11c\ube44\uc2a4","rel":"","context":"&quot;IT&quot;\uc5d0\uc11c","block_context":{"text":"IT","link":"https:\/\/nori.company\/?cat=138"},"img":{"alt_text":"","src":"","width":0,"height":0},"classes":[]},{"id":25086,"url":"https:\/\/nori.company\/?p=25086","url_meta":{"origin":28287,"position":5},"title":"\uba54\ud0c0 \uc800\ucee4\ubc84\uadf8, &#8220;\ub300\ud654\ud615 \uc0dd\uc131 AI \ud310 \ubc14\uafbc\ub2e4!&#8221;&#8230;&#8217;7\uc5b5\u00b713\uc5b5\u00b770\uc5b5 \uac1c\uc758 \uac01 \ud30c\ub77c\ubbf8\ud130\uc758 \uc0ac\uc804\ud559\uc2b5\ubaa8\ub378 \u2018\ub77c\ub9c8 2&#8242; \uc624\ud508\uc18c\uc2a4\ub85c \uacf5\uac1c","author":"\ub178\ub9ac\ucef4\ud37c\ub2c8","date":"2023\ub144 7\uc6d4 19\uc77c","format":false,"excerpt":"\uba54\ud0c0\uac00 \ucc28\uc138\ub300 \uc778\uacf5\uc9c0\ub2a5(AI) \ub300\uaddc\ubaa8 \uc5b8\uc5b4 \ubaa8\ub378(LLM) '\ub77c\ub9c82( LLaMa2)'\ub97c \uc624\ud508 \uc18c\uc2a4\ub85c 18\uc77c(\ud604\uc9c0\uc2dc\uac04), \uacf5\uac1c\ud588\ub2e4.\ubaa8\ub378\uc740 7\uc5b5, 13\uc5b5, 70\uc5b5 \uac1c\uc758 \ud30c\ub77c\ubbf8\ud130\ub85c \uc0ac\uc804 \ud559\uc2b5\ub418\uace0 \ubbf8\uc138 \uc870\uc815\ub41c \ubaa8\ub378\ub85c 2\uc870 \uac1c\uc758 \ud1a0\ud070\uc73c\ub85c \ud6c8\ub828\ub418\uc5c8\uc73c\uba70 \uc774\uc804 \ubaa8\ub378 'Llama 1'\ubcf4\ub2e4 \ucee8\ud14d\uc2a4\ud2b8 \uae38\uc774\uac00 \ub450 \ubc30\uc774\uba70, 100\ub9cc \uac1c \uc774\uc0c1\uc758 \uc778\uac04 \uc8fc\uc11d\uc73c\ub85c \ud6c8\ub828\ub418\uc5c8\ub2e4.\uba54\ud0c0\ub294 \uc774\ub0a0 \uacf5\uac1c\uc640 \ub354\ubd88\uc5b4 \"\uc6b0\ub9ac\ub294 \uc774\ub7ec\ud55c \ub300\uaddc\ubaa8 \uc5b8\uc5b4 \ubaa8\ub378\uc5d0 \ud798\uc744 \ubc1c\ud718\ud558\uace0\u2026","rel":"","context":"&quot;AI&quot;\uc5d0\uc11c","block_context":{"text":"AI","link":"https:\/\/nori.company\/?cat=155"},"img":{"alt_text":"","src":"","width":0,"height":0},"classes":[]}],"_links":{"self":[{"href":"https:\/\/nori.company\/index.php?rest_route=\/wp\/v2\/posts\/28287","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/nori.company\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/nori.company\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/nori.company\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/nori.company\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=28287"}],"version-history":[{"count":1,"href":"https:\/\/nori.company\/index.php?rest_route=\/wp\/v2\/posts\/28287\/revisions"}],"predecessor-version":[{"id":28288,"href":"https:\/\/nori.company\/index.php?rest_route=\/wp\/v2\/posts\/28287\/revisions\/28288"}],"wp:attachment":[{"href":"https:\/\/nori.company\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=28287"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/nori.company\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=28287"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/nori.company\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=28287"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}