{"id":22007,"date":"2023-04-03T11:48:34","date_gmt":"2023-04-03T02:48:34","guid":{"rendered":"http:\/\/nori.company\/?p=22007"},"modified":"2023-04-03T11:48:34","modified_gmt":"2023-04-03T02:48:34","slug":"lit-llama-nanogpt-%ea%b8%b0%eb%b0%98-%ec%98%a4%ed%94%88%ec%86%8c%ec%8a%a4-llama-%ea%b5%ac%ed%98%84%ec%b2%b4","status":"publish","type":"post","link":"https:\/\/nori.company\/?p=22007","title":{"rendered":"Lit-LLaMA &#8211; nanoGPT \uae30\ubc18 \uc624\ud508\uc18c\uc2a4 LLaMA \uad6c\ud604\uccb4"},"content":{"rendered":"<ul>\n<li>GPL \ub77c\uc774\uc13c\uc2a4\uc778 LLaMA \ucf54\ub4dc \ub300\uc2e0, nanoGPT \uae30\ubc18\uc73c\ub85c Apache 2.0 \ub77c\uc774\uc13c\uc2a4\ub85c \uc644\uc804\ud788 \ubcc4\ub3c4 \uac1c\ubc1c\ub41c \uc624\ud508\uc18c\uc2a4<\/li>\n<li>bolierplate \uc5c6\uc774 \ub2e8\uc77c \ud30c\uc77c\ub85c \uad6c\ud604\ub418\uc5b4 \uc2ec\ud50c\ud558\uba70, \uae30\uc874 \ubaa8\ub378\uacfc \uc218\uce58\uc801\uc73c\ub85c \ub3d9\uc77c<\/li>\n<li>FlashAttention, \uc591\uc790\ud654, LoRA \ud30c\uc778\ud29c\ub2dd, \uc0ac\uc804 \ud559\uc2b5\ub4f1\uc744 \uc9c0\uc6d0<\/li>\n<li>\uc18c\ube44\uc790\uc6a9 \ud558\ub4dc\uc6e8\uc5b4\uc5d0\uc11c \uc2e4\ud589\ub418&#8230;\n<p>\ucd9c\ucc98 : <a href=\"https:\/\/news.hada.io\/topic?id=8865\" target=\"_blank\" rel=\"noopener\">GeekNews &#8211; \uac1c\ubc1c\/\uae30\uc220\/\uc2a4\ud0c0\ud2b8\uc5c5 \ub274\uc2a4 \uc11c\ube44\uc2a4 <\/a><\/li>\n<\/ul>\n","protected":false},"excerpt":{"rendered":"<p>GPL \ub77c\uc774\uc13c\uc2a4\uc778 LLaMA \ucf54\ub4dc \ub300\uc2e0, nanoGPT \uae30\ubc18\uc73c\ub85c Apache 2.0 \ub77c\uc774\uc13c\uc2a4\ub85c \uc644\uc804\ud788 \ubcc4\ub3c4 \uac1c\ubc1c\ub41c \uc624\ud508\uc18c\uc2a4 bolierplate \uc5c6\uc774 \ub2e8\uc77c \ud30c\uc77c\ub85c \uad6c\ud604\ub418\uc5b4 \uc2ec\ud50c\ud558\uba70,&hellip;<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"jetpack_post_was_ever_published":false,"_jetpack_newsletter_access":"","_jetpack_dont_email_post_to_subs":false,"_jetpack_newsletter_tier_id":0,"_jetpack_memberships_contains_paywalled_content":false,"_jetpack_memberships_contains_paid_content":false,"footnotes":"","jetpack_publicize_message":"","jetpack_publicize_feature_enabled":true,"jetpack_social_post_already_shared":true,"jetpack_social_options":{"image_generator_settings":{"template":"highway","default_image_id":0,"font":"","enabled":false},"version":2}},"categories":[138],"tags":[163],"class_list":["post-22007","post","type-post","status-publish","format-standard","hentry","category-it","tag-geeknews-----"],"jetpack_publicize_connections":[],"jetpack_featured_media_url":"","jetpack_sharing_enabled":true,"jetpack_shortlink":"https:\/\/wp.me\/pa3PO4-5IX","jetpack-related-posts":[{"id":25088,"url":"https:\/\/nori.company\/?p=25088","url_meta":{"origin":22007,"position":0},"title":"Meta, \ucc28\uc138\ub300 \uc5b8\uc5b4\ubaa8\ub378 Llama 2 \uacf5\uac1c","author":"\ub178\ub9ac\ucef4\ud37c\ub2c8","date":"2023\ub144 7\uc6d4 19\uc77c","format":false,"excerpt":"\uc5f0\uad6c \ubc0f \uc0c1\uc5c5\uc6a9\ub3c4 \ubaa8\ub450 \uc0ac\uc6a9\uac00\ub2a5\ud55c \uc624\ud508\uc18c\uc2a4 \ub300\uaddc\ubaa8 \uc5b8\uc5b4\ubaa8\ub378 Llama 1 \ub300\ube44 40% \ub9ce\uc740 2\uc870\uac1c\uc758 \ud1a0\ud070 \ub370\uc774\ud130\ub85c \ud6c8\ub828 2\ubc30\uc758 \ucee8\ud14d\uc2a4\ud2b8 \uae38\uc774: 4096 \uc138\uac00\uc9c0 \ud06c\uae30\uc758 \ubaa8\ub378: 7B, 13B, 70B \ucd94\ub860, \ucf54\ub529, \uc219\ub828\ub3c4, \uc9c0\uc2dd\ud14c\uc2a4\ud2b8\ub4f1 \ub9ce\uc740 \ubca4\uce58\ub9c8\ud06c\uc5d0\uc11c \ub2e4\ub978 \uc624\ud508\uc18c\uc2a4 \uc5b8\uc5b4 \ubaa8\ub378\ubcf4\ub2e4 \ud6cc\ub96d\ud55c ... \ucd9c\ucc98 : GeekNews - \uac1c\ubc1c\/\uae30\uc220\/\uc2a4\ud0c0\ud2b8\uc5c5 \ub274\uc2a4 \uc11c\ube44\uc2a4","rel":"","context":"&quot;IT&quot;\uc5d0\uc11c","block_context":{"text":"IT","link":"https:\/\/nori.company\/?cat=138"},"img":{"alt_text":"","src":"","width":0,"height":0},"classes":[]},{"id":25092,"url":"https:\/\/nori.company\/?p=25092","url_meta":{"origin":22007,"position":1},"title":"Meta, \ucc28\uc138\ub300 \uc5b8\uc5b4\ubaa8\ub378 Llama 2 \uacf5\uac1c","author":"\ub178\ub9ac\ucef4\ud37c\ub2c8","date":"2023\ub144 7\uc6d4 19\uc77c","format":false,"excerpt":"\uc5f0\uad6c \ubc0f \uc0c1\uc5c5\uc6a9\ub3c4 \ubaa8\ub450 \uc0ac\uc6a9\uac00\ub2a5\ud55c \uc624\ud508\uc18c\uc2a4 \ub300\uaddc\ubaa8 \uc5b8\uc5b4\ubaa8\ub378 Llama 1 \ub300\ube44 40% \ub9ce\uc740 2\uc870\uac1c\uc758 \ud1a0\ud070 \ub370\uc774\ud130\ub85c \ud6c8\ub828 2\ubc30\uc758 \ucee8\ud14d\uc2a4\ud2b8 \uae38\uc774: 4096 \uc138\uac00\uc9c0 \ud06c\uae30\uc758 \ubaa8\ub378: 7B, 13B, 70B \ucd94\ub860, \ucf54\ub529, \uc219\ub828\ub3c4, \uc9c0\uc2dd\ud14c\uc2a4\ud2b8\ub4f1 \ub9ce\uc740 \ubca4\uce58\ub9c8\ud06c\uc5d0\uc11c \ub2e4\ub978 \uc624\ud508\uc18c\uc2a4 \uc5b8\uc5b4 \ubaa8\ub378\ubcf4\ub2e4 \ud6cc\ub96d\ud55c ... \ucd9c\ucc98 : GeekNews - \uac1c\ubc1c\/\uae30\uc220\/\uc2a4\ud0c0\ud2b8\uc5c5 \ub274\uc2a4 \uc11c\ube44\uc2a4","rel":"","context":"&quot;IT&quot;\uc5d0\uc11c","block_context":{"text":"IT","link":"https:\/\/nori.company\/?cat=138"},"img":{"alt_text":"","src":"","width":0,"height":0},"classes":[]},{"id":25959,"url":"https:\/\/nori.company\/?p=25959","url_meta":{"origin":22007,"position":2},"title":"\uc624\ud508\uc18c\uc2a4 \uc5b8\uc5b4 \ubaa8\ub378\uc758 \ud604\uc7ac","author":"\ub178\ub9ac\ucef4\ud37c\ub2c8","date":"2023\ub144 8\uc6d4 9\uc77c","format":false,"excerpt":"LLaMA 2 \ucd9c\uc2dc\ud6c4 \ub354 \ub728\uac70\uc6cc\uc9d0 \uac70\uc758 \ubaa8\ub4e0 \uc624\ud508\uc18c\uc2a4 \ubaa8\ub378 \uadf8\ub8f9\ub4e4\uc774 \uc0c8\ub85c\uc6b4 \ubca0\uc774\uc2a4 \ubaa8\ub378\ub85c \uc790\uc2e0\uc758 \ubaa8\ub378\uc744 \uc5c5\ub370\uc774\ud2b8 WizardLM, Airoboros, Hermes \ub4f1 \ud604\uc7ac \uac00\uc7a5 \uac15\ub825\ud55c \ubaa8\ub378\uc740 StabilityAI\uc758 Stable Beluga 2 Llama2 70B \ubaa8\ub378\uc744 Orca \uc2a4\ud0c0\uc77c \ub370\uc774\ud130\uc14b\uc73c\ub85c \ud30c\uc778 \ud29c\ub2dd... \ucd9c\ucc98 : GeekNews - \uac1c\ubc1c\/\uae30\uc220\/\uc2a4\ud0c0\ud2b8\uc5c5 \ub274\uc2a4 \uc11c\ube44\uc2a4","rel":"","context":"&quot;IT&quot;\uc5d0\uc11c","block_context":{"text":"IT","link":"https:\/\/nori.company\/?cat=138"},"img":{"alt_text":"","src":"","width":0,"height":0},"classes":[]},{"id":23067,"url":"https:\/\/nori.company\/?p=23067","url_meta":{"origin":22007,"position":3},"title":"MPT-7B, \uc0c1\uc5c5\uc801\uc73c\ub85c \uc774\uc6a9 \uac00\ub2a5\ud55c \uc624\ud508\uc18c\uc2a4 LLM","author":"\ub178\ub9ac\ucef4\ud37c\ub2c8","date":"2023\ub144 5\uc6d4 9\uc77c","format":false,"excerpt":"1\uc870\uac1c\uc758 \ud1a0\ud070\uc73c\ub85c \ud559\uc2b5\ub41c \ud2b8\ub79c\uc2a4\ud3ec\uba38 LLaMA-7B\uc640 \ube44\uc2b7\ud55c \ud488\uc9c8\uc774\uc9c0\ub9cc \uc624\ud508\uc18c\uc2a4, \uc0c1\uc5c5\uc801 \uc774\uc6a9 \uac00\ub2a5 \ucd94\uac00\ub85c 3\uac1c\uc758 \ud30c\uc778\ud29c\ub2dd \ubaa8\ub378\ub3c4 \uacf5\uac1c MPT-7B-Instruct, MPT-7B-Chat, MPT-7B-StoryWriter-65K+(65K \ud1a0\ud070 \uc9c0\uc6d0) MosaicML \ud50c\ub7ab\ud3fc\uc5d0\uc11c 9.5\uc77c\uac04 \uc57d ~$200K \ube44\uc6a9\uc73c\ub85c... \ucd9c\ucc98 : GeekNews - \uac1c\ubc1c\/\uae30\uc220\/\uc2a4\ud0c0\ud2b8\uc5c5 \ub274\uc2a4 \uc11c\ube44\uc2a4","rel":"","context":"&quot;IT&quot;\uc5d0\uc11c","block_context":{"text":"IT","link":"https:\/\/nori.company\/?cat=138"},"img":{"alt_text":"","src":"","width":0,"height":0},"classes":[]},{"id":22475,"url":"https:\/\/nori.company\/?p=22475","url_meta":{"origin":22007,"position":4},"title":"RedPajama &#8211; LLaMA \ub370\uc774\ud130\uc14b\uc744 \uc7ac\uc791\uc131\ud558\ub294 \uc624\ud508\uc18c\uc2a4 \ud504\ub85c\uc81d\ud2b8","author":"\ub178\ub9ac\ucef4\ud37c\ub2c8","date":"2023\ub144 4\uc6d4 19\uc77c","format":false,"excerpt":"LLaMA, Alpaca, Vicuna \uac19\uc740 \ubc18\uac1c\ubc29\ud615 \ubaa8\ub378\uc774 \uc544\ub2c8\ub77c \uc7ac\ud604\uac00\ub2a5\ud558\uace0 \uc644\uc804\ud55c \uac1c\ubc29\ud615 \uc5b8\uc5b4 \ubaa8\ub378\uc744 \ub9cc\ub4e4\uae30 \uc704\ud55c \ud504\ub85c\uc81d\ud2b8 3\uac00\uc9c0 \uad6c\uc131\uc694\uc18c \ub192\uc740 \ud488\uc9c8\uacfc \ub113\uc740 \ucee4\ubc84\ub9ac\uc9c0\ub97c \uac00\uc9c4 Pre-Training \ub370\uc774\ud130 \uc774 \ub370\uc774\ud130 \uae30\ubc18\uc73c\ub85c \ub300\uaddc\ubaa8\ub85c \ud559\uc2b5\ub41c \ubca0\uc774\uc2a4 \ubaa8\ub378 \ubca0\uc774\uc2a4 \ubcf4\ub378\uc740 \uc548\uc804\ud558\uace0 \uc0ac\uc6a9... \ucd9c\ucc98 : GeekNews - \uac1c\ubc1c\/\uae30\uc220\/\uc2a4\ud0c0\ud2b8\uc5c5 \ub274\uc2a4 \uc11c\ube44\uc2a4","rel":"","context":"&quot;IT&quot;\uc5d0\uc11c","block_context":{"text":"IT","link":"https:\/\/nori.company\/?cat=138"},"img":{"alt_text":"","src":"","width":0,"height":0},"classes":[]},{"id":23242,"url":"https:\/\/nori.company\/?p=23242","url_meta":{"origin":22007,"position":5},"title":"Open-LLaMa &#8211; LLM \uc791\uc131\uc6a9 \uc644\uc804\ud55c \ud2b8\ub808\uc774\ub2dd \ud30c\uc774\ud504\ub77c\uc778","author":"\ub178\ub9ac\ucef4\ud37c\ub2c8","date":"2023\ub144 5\uc6d4 16\uc77c","format":false,"excerpt":"\ub370\uc774\ud130\uc14b \uc0dd\uc131\ubd80\ud130 \ud1a0\ud070\ud654, \ud504\ub86c\ud504\ud2b8 \ud29c\ub2dd, LoRA \ubc0f RLHF \uae4c\uc9c0 \uc804\uccb4 \ud30c\uc774\ud504\ub77c\uc778\uc744 \uad6c\uc131\ud55c \uc624\ud508\uc18c\uc2a4 \uc0ac\uc804\ud559\uc2b5\ub41c Open-LLama-V2-pretrain \ubaa8\ub378\ub3c4 Hugging Face\uc5d0 \uacf5\uac1c FastChat \ud3c9\uac00\ubc29\ubc95\uc5d0 \uc758\ud558\uba74 GPT-3.5\uc640 \ube44\uad50\uc2dc \uc57d 89%\uc758 \uc131\ub2a5\uc744 \ub0b8\ub2e4\uace0(\uc911\uad6d\uc5b4\ub85c\ub41c \uc9c8\ubb38\uc5d0 \ub300\ud574) \ud559\uc2b5 \uc2a4\ud53c\ub4dc\ub294 36... \ucd9c\ucc98 : GeekNews - \uac1c\ubc1c\/\uae30\uc220\/\uc2a4\ud0c0\ud2b8\uc5c5 \ub274\uc2a4 \uc11c\ube44\uc2a4","rel":"","context":"&quot;IT&quot;\uc5d0\uc11c","block_context":{"text":"IT","link":"https:\/\/nori.company\/?cat=138"},"img":{"alt_text":"","src":"","width":0,"height":0},"classes":[]}],"_links":{"self":[{"href":"https:\/\/nori.company\/index.php?rest_route=\/wp\/v2\/posts\/22007","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/nori.company\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/nori.company\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/nori.company\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/nori.company\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=22007"}],"version-history":[{"count":1,"href":"https:\/\/nori.company\/index.php?rest_route=\/wp\/v2\/posts\/22007\/revisions"}],"predecessor-version":[{"id":22008,"href":"https:\/\/nori.company\/index.php?rest_route=\/wp\/v2\/posts\/22007\/revisions\/22008"}],"wp:attachment":[{"href":"https:\/\/nori.company\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=22007"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/nori.company\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=22007"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/nori.company\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=22007"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}